# !/usr/bin/env python
# -*- coding:utf-8 -*-
# project name: agent_interface_automation
# author: "Lei Yong"
# creation time: 2020-11-26 14:33
# Email: leiyong711@163.com

import re
import json
import gzip
import time
import traceback
import mitmproxy.http
# from mitmproxy import ctx
from proxy_db import DB
from json_to_jsonSchema import to_jsonschema


COMPILE_IP_PATTERN = re.compile(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b")         # 客户端ip正则
COMPILE_FILTER_URL_PATTERN = re.compile(r'(.*?)cloudmall.(.*?)/api/(.*?)')    # 监听域名
COMPILE_BY_BANNER_PATTERN = re.compile(r'by_banner/([A-Za-z0-9]{20,})')       # by banner id 正则
COMPILE_BANNER_PATTERN = re.compile(r'(.*?)banner/(.*?)/(.*?)')               # banner_id 正则
COMPILE_ITEM_ID_PATTERN = re.compile(r'[0-9]{7,}')                            # item_id 正则

db = DB()

# 抓包设备ip
LISTEN_IP = [
        "127.0.0.1",
        "10.0.0.68",
        "10.0.0.85",
        "10.0.0.47",
]

# 无需入库的URL
EXCLUDE_URL = [
    'https://res.cloudmall.co/api/imp',
    "https://web.cloudmall.co/api/v/4.7.0/homepage/banner/ad?language=zh-Hans&region=US&client=web_4.7.1&callback"
    "=jsonp_0",
    "https://web.cloudmall.co/api/v/4.7.0/comment/item/39907749?language=zh-Hans&region=US&client=web_4.7.1&page=0"
    "&limit=200&callback=jsonp_0",
]

# 需要抓取的资源类型
SEC_FETCH_DEST = [
    "empty",  # 一般请求
    # "image",   # 图片资源
    # "script",  # js资源
    # "style",   # 样式资源
    # "iframe",  # 页面资源
]


# 检查是否属于监听客户端及URL地址
def is_monitoring_data(flow):

    # 请求的URL
    proxy_url = flow.request.url

    # 不是所需的请求资源类型
    fetch_dest = flow.request.headers.get('sec-fetch-dest')
    if not fetch_dest in SEC_FETCH_DEST:
        return False, None

    # 不需要抓包的URL
    filter_urls = COMPILE_FILTER_URL_PATTERN.findall(proxy_url)
    if not filter_urls:
        return False, None

    # URL为需排除的URL地址
    if proxy_url in EXCLUDE_URL:
        return False, None

    # 连接的客户端IP
    client_ip = COMPILE_IP_PATTERN.findall(str(flow.client_conn))[0]

    # 是否属于监控的IP
    if client_ip in LISTEN_IP:
        return True, proxy_url
    return False, None


def url_parameterization(url):
    """
    url参数化
    :param url: URL接口
    :return: 参数化后的URL
    """
    url = url.split('?')[0]

    # 参数化banner_id
    # https://web.cloudmall.co/api/v/4.7.0/activity/by_banner/5fae5f23f08a5b654d0eeab9
    banner_end = COMPILE_BY_BANNER_PATTERN.findall(url)
    if len(banner_end) > 0:
        db.inset_banner_id(banner_id=banner_end[0])
        return url.replace(banner_end[0], "{banner_id}")

    # 参数化banner_id
    # https://web.cloudmall.co/api/v/4.7.0/homepage/banner/5fae5f23f08a5b654d0eeab9/items
    banner = COMPILE_BANNER_PATTERN.findall(url)
    if len(banner) > 0:
        db.inset_banner_id(banner_id=banner[0][1])
        return url.replace(banner[0][1], "{banner_id}")

    # 参数化item_id
    # https://web.cloudmall.co/api/v/4.7.0/activity/by_item/35186425
    item_id = COMPILE_ITEM_ID_PATTERN.findall(url)
    if len(item_id) > 0:
        db.inset_item_id(item_id=item_id[0])
        return url.replace(item_id[0], "{item_id}")
    return url


# 请求
def request(flow: mitmproxy.http.HTTPFlow):
    pass
    # status, proxy_url = is_monitoring_data(flow)
    # if status:
    #     print("请求的URL: {}".format(proxy_url))
    #     db.inset(proxy_url)


# 响应
def response(flow: mitmproxy.http.HTTPFlow):
    status, proxy_url = is_monitoring_data(flow)  # 检查是否为想抓取的数据
    request_method = flow.request.method  # 请求方法
    # print("请求类型",flow.request.headers.get('sec-fetch-dest'))

    # 排除预检请求
    if status and request_method != "OPTIONS":

        url = url_parameterization(proxy_url)  # url 参数化
        method = flow.request.method           # 请求方法
        headers = dict(flow.request.headers)   # 请求头
        request_data = ""                      # 请求参数默认值

        timer = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
        print(f'\n\n-------------{timer}-------------')
        print(f'请求URL: {url}')

        # 请求参数
        if request_method == 'GET':
            request_data = dict(flow.request.query)
        elif request_method == 'POST':
            try:
                request_data = json.loads(flow.request.data.content.decode("utf-8"))
            except Exception as e:
                print(f'\033[1;31m POST 转码失败   {flow.request.data.content.decode("utf-8")} \033[0m')
                request_data = flow.request.data.content.decode("utf-8")

        # 响应数据解码
        res = flow.response.data.content
        try:
            res = gzip.decompress(res)
            response_result = json.loads(res.decode("utf-8"))
        except Exception as e:
            print(f'\033[1;31m 转码失败原因 {e} \033[0m')
            print(f'\033[1;31m 响应 转码失败 {res.decode("utf-8")} \033[0m')
            try:
                response_result = eval(res.decode("utf-8"))
            except NameError:
                response_result = res.decode("utf-8")

        print(f'请求方式: {method}')
        # print(f'请求参数: {request_data}')
        # print(f'请求参数类型{type(request_data)}')
        # print(f'请求headers: {headers}')
        # print(f'响应数据: {response_result}')
        # print(f'响应数据类型: {type(response_result)}')

        # json响应转jsonSchema验证模型
        struct_list = []
        struct_verify = ''
        try:
            to_jsonschema(response_result, struct_list)
            for i in struct_list:
                struct_verify += i
        except Exception as e:
            print(f'转化验证模型失败: {e}')
            struct_verify = ''
            traceback.print_exc()

        # 插入接口信息到数据库
        try:
            db.inset_interface(url=url, method=method, request_data=request_data, headers=headers,
                               response_result=str(response_result), struct_verify=struct_verify)
        except Exception as e:
            print(f'错误原因: {e}')
            print(f'请求参数: {request_data}')
            print(f'请求参数类型{type(request_data)}')
            # print(f'请求headers: {headers}')
            print(f'响应数据: {response_result}')
            print(f'响应数据类型: {type(response_result)}')
        print('--------------- END ------------------------\n\n')


# 安装 pip3 install mitmproxy
# 启动命令 --web-host 指定查看web页面ip --web-port 指定web页面端口  -p 指定监听端口  -s 指定扩展脚本
# mitmweb --web-host 0.0.0.0 --web-port 80 -p 8888 -s proxy.py
# web 正则查找
# ~m .*(get|post)+.* & ~u https://(.*?).cloudmall.(.+?)/api/(.+?)
